single-machine-performance-regression_detector:
    allow_failure: false
    artifacts:
      expire_in: 1 weeks
      paths:
      - submission_metadata
      - ${CI_COMMIT_SHA}-baseline_sha
      - outputs/report.md
      - outputs/regression_signal.json
      - outputs/bounds_check_signal.json
      - outputs/junit.xml
      - outputs/report.json
      - outputs/decision_record.md
      when: always
    image: registry.ddbuild.io/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
    needs:
    - artifacts: false
      job: single_machine_performance-amd64-a7
    rules:
    - if: $CI_COMMIT_BRANCH == "main"
      when: never
    - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.[0-9]+\.x$/
      when: never
    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
      when: never
    - when: on_success
    script:
    - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2"
      token)" || exit $?; export DATADOG_API_KEY
    - datadog-ci tag --level job --tags smp_failure_mode:"unknown"
    - mkdir outputs
    - git fetch origin
    - SMP_BASE_BRANCH=$(inv release.get-release-json-value base_branch --no-worktree)
    - echo "Looking for merge base for branch ${SMP_BASE_BRANCH}"
    - SMP_MERGE_BASE=$(git merge-base ${CI_COMMIT_SHA} origin/${SMP_BASE_BRANCH})
    - echo "Merge base is ${SMP_MERGE_BASE}"
    - AWS_NAMED_PROFILE="single-machine-performance"
    - SMP_ACCOUNT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT account_id)
      || exit $?
    - SMP_ECR_URL=${SMP_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com
    - SMP_AGENT_TEAM_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT agent_team_id)
      || exit $?
    - SMP_API=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT api_url) || exit
      $?
    - SMP_BOT_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT bot_login)
      || exit $?
    - SMP_BOT_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SMP_ACCOUNT bot_token)
      || exit $?
    - aws configure set aws_access_key_id "$SMP_BOT_ID" --profile ${AWS_NAMED_PROFILE}
    - aws configure set aws_secret_access_key "$SMP_BOT_KEY" --profile ${AWS_NAMED_PROFILE}
    - aws configure set region us-west-2 --profile ${AWS_NAMED_PROFILE}
    - aws --profile single-machine-performance s3 cp s3://smp-cli-releases/v${SMP_VERSION}/x86_64-unknown-linux-gnu/smp
      smp
    - chmod +x smp
    - BASELINE_SHA="${SMP_MERGE_BASE}"
    - echo "Computing baseline..."
    - echo "Checking if commit ${BASELINE_SHA} is recent enough..."
    - FOUR_DAYS_BEFORE_NOW=$(date --date="-4 days +1 hour" "+%s")
    - BASELINE_COMMIT_TIME=$(git -c log.showSignature=false show --no-patch --format=%ct
      ${BASELINE_SHA})
    - "if [[ ${BASELINE_COMMIT_TIME} -le ${FOUR_DAYS_BEFORE_NOW} ]]\nthen\n    echo\
      \ \"ERROR: Merge-base of this branch is too old for SMP. Please update your branch\
      \ by merging an up-to-date main branch into your branch or by rebasing it on an\
      \ up-to-date main branch.\"\n    datadog-ci tag --level job --tags smp_failure_mode:\"\
      merge-base-too-old\"\n    exit 1\nfi\n"
    - echo "Commit ${BASELINE_SHA} is recent enough"
    - echo "Checking if image exists for commit ${BASELINE_SHA}..."
    - "while [[ ! $(aws ecr describe-images --region us-west-2 --profile single-machine-performance\
      \ --registry-id \"${SMP_ACCOUNT_ID}\" --repository-name \"${SMP_AGENT_TEAM_ID}-agent\"\
      \ --image-ids imageTag=\"${BASELINE_SHA}-7-amd64\") ]]\ndo\n    echo \"No image\
      \ exists for ${BASELINE_SHA} - checking predecessor of ${BASELINE_SHA} next\"\n\
      \    BASELINE_SHA=$(git rev-parse ${BASELINE_SHA}^)\n    echo \"Checking if commit\
      \ ${BASELINE_SHA} is recent enough...\"\n    BASELINE_COMMIT_TIME=$(git -c log.showSignature=false\
      \ show --no-patch --format=%ct ${BASELINE_SHA})\n    if [[ ${BASELINE_COMMIT_TIME}\
      \ -le ${FOUR_DAYS_BEFORE_NOW} ]]\n    then\n        echo \"ERROR: Merge-base of\
      \ this branch is too old for SMP. Please update your branch by merging an up-to-date\
      \ main branch into your branch or by rebasing it on an up-to-date main branch.\"\
      \n        datadog-ci tag --level job --tags smp_failure_mode:\"merge-base-too-old-predecessor\"\
      \n        exit 1\n    fi\n    echo \"Commit ${BASELINE_SHA} is recent enough\"\
      \n    echo \"Checking if image exists for commit ${BASELINE_SHA}...\"\ndone\n"
    - echo "Image exists for commit ${BASELINE_SHA}"
    - echo "Baseline SHA is ${BASELINE_SHA}"
    - echo -n "${BASELINE_SHA}" > "${CI_COMMIT_SHA}-baseline_sha"
    - aws s3 cp --profile single-machine-performance --only-show-errors "${CI_COMMIT_SHA}-baseline_sha"
      "s3://${SMP_AGENT_TEAM_ID}-smp-artifacts/information/"
    - BASELINE_IMAGE=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${BASELINE_SHA}-7-amd64
    - echo "${BASELINE_SHA} | ${BASELINE_IMAGE}"
    - COMPARISON_IMAGE=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${CI_COMMIT_SHA}-7-amd64
    - echo "${CI_COMMIT_SHA} | ${COMPARISON_IMAGE}"
    - SMP_TAGS="ci_pipeline_id=${CI_PIPELINE_ID},ci_job_id=${CI_JOB_ID}"
    - echo "Tags passed through SMP are ${SMP_TAGS}"
    - RUST_LOG="info,aws_config::profile::credentials=error"
    - RUST_LOG_DEBUG="debug,aws_config::profile::credentials=error"
    - "RUST_LOG=\"${RUST_LOG}\" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API}\
      \ --aws-named-profile ${AWS_NAMED_PROFILE} \\\njob submit \\\n--baseline-image\
      \ ${BASELINE_IMAGE} \\\n--comparison-image ${COMPARISON_IMAGE} \\\n--baseline-sha\
      \ ${BASELINE_SHA} \\\n--comparison-sha ${CI_COMMIT_SHA} \\\n--target-config-dir\
-     \ test/regression/ \\\n--submission-metadata submission_metadata \\\n--tags ${SMP_TAGS}\
?                                                                              ^ ^^^^^^^^^^^^
+     \ test/regression/ \\\n--submission-metadata submission_metadata \\\n--total-samples\
?                                                                             ++ ^^ ^^^^^^
-     \ || {\n  exit_code=$?\n  echo \"smp job submit command failed with code $exit_code\"\
-     \n  datadog-ci tag --level job --tags smp_failure_mode:\"job-submission\"\n  exit\
-     \ $exit_code\n}\n"
+     \ 1800\n--replicas 50 \\\n--tags ${SMP_TAGS} || {\n  exit_code=$?\n  echo \"smp\
+     \ job submit command failed with code $exit_code\"\n  datadog-ci tag --level job\
+     \ --tags smp_failure_mode:\"job-submission\"\n  exit $exit_code\n}\n"
    - SMP_JOB_ID=$(jq -r '.jobId' submission_metadata)
    - echo "SMP Job Id is ${SMP_JOB_ID}"
    - datadog-ci tag --level job --tags smp_job_id:${SMP_JOB_ID}
    - "RUST_LOG=\"${RUST_LOG}\" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API}\
      \ --aws-named-profile ${AWS_NAMED_PROFILE} \\\njob status \\\n--wait \\\n--wait-delay-seconds\
      \ 60 \\\n--submission-metadata submission_metadata || {\n  exit_code=$?\n  echo\
      \ \"smp job status command failed with code $exit_code\"\n  datadog-ci tag --level\
      \ job --tags smp_failure_mode:\"job-status\"\n  exit $exit_code\n}\n"
    - "RUST_LOG=\"${RUST_LOG}\" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API}\
      \ --aws-named-profile ${AWS_NAMED_PROFILE} \\\njob sync \\\n--submission-metadata\
      \ submission_metadata \\\n--output-path outputs || {\n  exit_code=$?\n  echo \"\
      smp job sync command failed with code $exit_code\"\n  datadog-ci tag --level job\
      \ --tags smp_failure_mode:\"job-sync\"\n  exit $exit_code\n}\n"
    - cat outputs/report.md | sed "s/^\$/$(echo -ne '\uFEFF\u00A0\u200B')/g"
    - datadog-ci junit upload --service datadog-agent outputs/junit.xml
    - datadog-ci tag --level job --tags smp_failure_mode:"none"
    - datadog-ci tag --level job --tags smp_optimization_goal:"passed"
    - "RUST_LOG=\"${RUST_LOG}\" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API}\
      \ --aws-named-profile ${AWS_NAMED_PROFILE} \\\n  job result \\\n  --submission-metadata\
      \ submission_metadata --signal regression-detector || {\n  exit_code=$?\n  echo\
      \ \"smp regression detector has detected a regression\"\n  datadog-ci tag --level\
      \ job --tags smp_optimization_goal:\"failed\"\n}\n"
    - datadog-ci tag --level job --tags smp_bounds_check:"passed"
    - "RUST_LOG=\"${RUST_LOG}\" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API}\
      \ --aws-named-profile ${AWS_NAMED_PROFILE} \\\n  job result \\\n  --submission-metadata\
      \ submission_metadata --signal bounds-check || {\n  exit_code=$?\n  echo \"smp\
      \ regression detector has detected a failed bounds check\"\n  datadog-ci tag --level\
      \ job --tags smp_bounds_check:\"failed\"\n}\n"
    - datadog-ci tag --level job --tags smp_quality_gates:"failed"
    - "python3 <<'EOF'\nimport json\nimport sys\n\ntry:\n    with open('outputs/report.json')\
      \ as f:\n        data = json.load(f)\nexcept FileNotFoundError:\n    print(\"\
      Machine readable report not found.\")\n    sys.exit(1)\nexcept json.JSONDecodeError\
      \ as e:\n    print(f\"Error parsing JSON report: {e}\")\n    sys.exit(1)\n\nexperiments\
      \ = data.get('experiments', {})\nfailed = False\ndecision_record = []\n\nfor exp_name,\
      \ exp_data in experiments.items():\n    if exp_name.startswith('quality_gate_'):\n\
      \        bounds_checks = exp_data.get('bounds_checks', {})\n        for check_name,\
      \ check_data in bounds_checks.items():\n            results = check_data.get('results',\
      \ {})\n            comparison = results.get('comparison', [])\n            num_total\
      \ = len(comparison)\n            failed_replicates = [\n                replicate\
      \ for replicate in comparison if not replicate.get('passed', False)\n        \
      \    ]\n            num_failed = len(failed_replicates)\n            num_passed\
      \ = num_total - num_failed\n            if failed_replicates:\n              \
      \  decision_record.append(\n                    f\"- **{exp_name}**, bounds check\
      \ **{check_name}**: {num_passed}/{num_total} replicas passed. Failed {num_failed}\
      \ which is > 0. Gate **FAILED**.\"\n                )\n                failed\
      \ = True\n            else:\n                decision_record.append(\n       \
      \             f\"- **{exp_name}**, bounds check **{check_name}**: {num_passed}/{num_total}\
      \ replicas passed. Gate passed.\"\n                )\n\nwith open('outputs/decision_record.md',\
      \ 'w') as f:\n    # Extra newline since this is appended to another report\n \
      \   f.write('\\n\\n## CI Pass/Fail Decision\\n\\n')\n    if failed:\n        f.write('\u274C\
      \ **Failed.** Some Quality Gates were violated.\\n\\n')\n        f.write('\\n'.join(decision_record))\n\
      \    else:\n        f.write('\u2705 **Passed.** All Quality Gates passed.\\n\\\
      n')\n        f.write('\\n'.join(decision_record))\n\nif failed:\n    print(\"\
      Quality gate failed, see decision record\")\n    sys.exit(1)\nelse:\n    print(\"\
      Quality gate passed.\")\n    sys.exit(0)\nEOF\n"
    - datadog-ci tag --level job --tags smp_quality_gates:"passed"
    stage: functional_test
    tags:
    - arch:amd64
    timeout: 1h10m
    variables:
      SMP_VERSION: 0.19.3